
\begin{longtable}{crllll}
\caption{\label{tab:function-task-static-defaults-all}Comparison of the success-rate, when the model converged, and the sparsity error, with 95\% confidence interval on the ``arithmetic datasets'' task. Each value is a summary of 100 different seeds.}\\
\toprule
\multicolumn{1}{c}{Op} & \multicolumn{1}{c}{Model} & \multicolumn{1}{c}{Success} & \multicolumn{2}{c}{Solved at} & \multicolumn{1}{c}{Sparsity error} \\
\cmidrule(l{3pt}r{3pt}){1-1} \cmidrule(l{3pt}r{3pt}){2-2} \cmidrule(l{3pt}r{3pt}){3-3} \cmidrule(l{3pt}r{3pt}){4-5} \cmidrule(l{3pt}r{3pt}){6-6}
 &  & Rate & Median & Mean & Mean\\
\midrule
\endfirsthead
\caption[]{Comparison of the success-rate, when the model converged, and the sparsity error, with 95\% confidence interval on the ``arithmetic datasets'' task. Each value is a summary of 100 different seeds. \textit{(continued)}}\\
\toprule
\multicolumn{1}{c}{Op} & \multicolumn{1}{c}{Model} & \multicolumn{1}{c}{Success} & \multicolumn{2}{c}{Solved at} & \multicolumn{1}{c}{Sparsity error} \\
\cmidrule(l{3pt}r{3pt}){1-1} \cmidrule(l{3pt}r{3pt}){2-2} \cmidrule(l{3pt}r{3pt}){3-3} \cmidrule(l{3pt}r{3pt}){4-5} \cmidrule(l{3pt}r{3pt}){6-6}
 &  & Rate & Median & Mean & Mean\\
\midrule
\endhead
\
\endfoot
\bottomrule
\endlastfoot
 & $\mathrm{NAC}_{\bullet,\mathrm{NMU}}$ & $93\% {~}^{+4\%}_{-7\%}$ & $1.8 \cdot 10^{6}$ & $2.0 \cdot 10^{6} {~}^{+1.0 \cdot 10^{5}}_{-9.7 \cdot 10^{4}}$ & $9.5 \cdot 10^{-7} {~}^{+4.2 \cdot 10^{-7}}_{-4.2 \cdot 10^{-7}}$\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet,\sigma}$ & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $2.5 \cdot 10^{6}$ & $2.6 \cdot 10^{6} {~}^{+8.8 \cdot 10^{4}}_{-7.2 \cdot 10^{4}}$ & $4.6 \cdot 10^{-5} {~}^{+5.0 \cdot 10^{-6}}_{-5.6 \cdot 10^{-6}}$\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet}$ & $31\% {~}^{+10\%}_{-8\%}$ & $2.8 \cdot 10^{6}$ & $3.0 \cdot 10^{6} {~}^{+2.9 \cdot 10^{5}}_{-2.4 \cdot 10^{5}}$ & $5.8 \cdot 10^{-4} {~}^{+4.8 \cdot 10^{-4}}_{-2.6 \cdot 10^{-4}}$\\

\nopagebreak
 & $\mathrm{NAC}_{+}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{Gated~}^{\mathrm{NAU}}_{\mathrm{NMU}}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & Linear & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NALU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NAU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NMU & $98\% {~}^{+1\%}_{-5\%}$ & $\mathbf{1.4 \cdot 10^{6}}$ & $\mathbf{1.5 \cdot 10^{6}} {~}^{+5.0 \cdot 10^{4}}_{-6.6 \cdot 10^{4}}$ & $\mathbf{4.2 \cdot 10^{-7}} {~}^{+2.9 \cdot 10^{-8}}_{-2.9 \cdot 10^{-8}}$\\

\nopagebreak
 & ReLU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
\multirow{-11}{*}{\centering\arraybackslash $\bm{\times}$} & ReLU6 & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\
\cmidrule{1-6}
 & $\mathrm{NAC}_{\bullet,\mathrm{NMU}}$ & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet,\sigma}$ & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet}$ & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{+}$ & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{Gated~}^{\mathrm{NAU}}_{\mathrm{NMU}}$ & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & Linear & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NALU & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NAU & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NMU & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & ReLU & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
\multirow{-11}{*}{\centering\arraybackslash $\bm{\mathbin{/}}$} & ReLU6 & $\mathbf{0\%} {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\
\cmidrule{1-6}
 & $\mathrm{NAC}_{\bullet,\mathrm{NMU}}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet,\sigma}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{+}$ & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $2.5 \cdot 10^{5}$ & $4.9 \cdot 10^{5} {~}^{+5.2 \cdot 10^{4}}_{-4.5 \cdot 10^{4}}$ & $2.3 \cdot 10^{-1} {~}^{+6.5 \cdot 10^{-3}}_{-6.5 \cdot 10^{-3}}$\\

\nopagebreak
 & $\mathrm{Gated~}^{\mathrm{NAU}}_{\mathrm{NMU}}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & Linear & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $6.1 \cdot 10^{4}$ & $\mathbf{6.3 \cdot 10^{4}} {~}^{+2.5 \cdot 10^{3}}_{-3.3 \cdot 10^{3}}$ & $2.5 \cdot 10^{-1} {~}^{+3.6 \cdot 10^{-4}}_{-3.6 \cdot 10^{-4}}$\\

\nopagebreak
 & NALU & $14\% {~}^{+8\%}_{-5\%}$ & $1.5 \cdot 10^{6}$ & $1.6 \cdot 10^{6} {~}^{+3.8 \cdot 10^{5}}_{-3.3 \cdot 10^{5}}$ & $1.7 \cdot 10^{-1} {~}^{+2.7 \cdot 10^{-2}}_{-2.5 \cdot 10^{-2}}$\\

\nopagebreak
 & NAU & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $\mathbf{1.8 \cdot 10^{4}}$ & $3.9 \cdot 10^{5} {~}^{+4.5 \cdot 10^{4}}_{-3.7 \cdot 10^{4}}$ & $\mathbf{3.2 \cdot 10^{-5}} {~}^{+1.3 \cdot 10^{-5}}_{-1.3 \cdot 10^{-5}}$\\

\nopagebreak
 & NMU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & ReLU & $62\% {~}^{+9\%}_{-10\%}$ & $6.2 \cdot 10^{4}$ & $7.6 \cdot 10^{4} {~}^{+8.3 \cdot 10^{3}}_{-7.0 \cdot 10^{3}}$ & $2.5 \cdot 10^{-1} {~}^{+2.4 \cdot 10^{-3}}_{-2.4 \cdot 10^{-3}}$\\

\nopagebreak
\multirow{-11}{*}{\centering\arraybackslash $\bm{+}$} & ReLU6 & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\
\cmidrule{1-6}
 & $\mathrm{NAC}_{\bullet,\mathrm{NMU}}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet,\sigma}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{+}$ & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $9.0 \cdot 10^{3}$ & $3.7 \cdot 10^{5} {~}^{+3.8 \cdot 10^{4}}_{-3.8 \cdot 10^{4}}$ & $2.3 \cdot 10^{-1} {~}^{+5.4 \cdot 10^{-3}}_{-5.4 \cdot 10^{-3}}$\\

\nopagebreak
 & $\mathrm{Gated~}^{\mathrm{NAU}}_{\mathrm{NMU}}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & Linear & $7\% {~}^{+7\%}_{-4\%}$ & $3.3 \cdot 10^{6}$ & $1.4 \cdot 10^{6} {~}^{+7.0 \cdot 10^{5}}_{-6.1 \cdot 10^{5}}$ & $1.8 \cdot 10^{-1} {~}^{+7.2 \cdot 10^{-2}}_{-5.8 \cdot 10^{-2}}$\\

\nopagebreak
 & NALU & $14\% {~}^{+8\%}_{-5\%}$ & $1.9 \cdot 10^{6}$ & $1.9 \cdot 10^{6} {~}^{+4.4 \cdot 10^{5}}_{-4.5 \cdot 10^{5}}$ & $2.1 \cdot 10^{-1} {~}^{+2.2 \cdot 10^{-2}}_{-2.2 \cdot 10^{-2}}$\\

\nopagebreak
 & NAU & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $\mathbf{5.0 \cdot 10^{3}}$ & $\mathbf{1.6 \cdot 10^{5}} {~}^{+1.7 \cdot 10^{4}}_{-1.6 \cdot 10^{4}}$ & $6.6 \cdot 10^{-2} {~}^{+2.5 \cdot 10^{-2}}_{-1.9 \cdot 10^{-2}}$\\

\nopagebreak
 & NMU & $56\% {~}^{+9\%}_{-10\%}$ & $1.0 \cdot 10^{6}$ & $1.0 \cdot 10^{6} {~}^{+5.8 \cdot 10^{2}}_{-5.8 \cdot 10^{2}}$ & $\mathbf{3.4 \cdot 10^{-4}} {~}^{+3.2 \cdot 10^{-5}}_{-2.6 \cdot 10^{-5}}$\\

\nopagebreak
 & ReLU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
\multirow{-11}{*}{\centering\arraybackslash $\bm{-}$} & ReLU6 & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\
\cmidrule{1-6}
 & $\mathrm{NAC}_{\bullet,\mathrm{NMU}}$ & $3\% {~}^{+5\%}_{-2\%}$ & $1.0 \cdot 10^{6}$ & $\mathbf{1.0 \cdot 10^{6}} {~}^{+NaN \cdot 10^{-Inf}}_{-NaN \cdot 10^{-Inf}}$ & $\mathbf{1.7 \cdot 10^{-1}} {~}^{+8.3 \cdot 10^{-3}}_{-8.1 \cdot 10^{-3}}$\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet,\sigma}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet}$ & $\mathbf{7\%} {~}^{+7\%}_{-4\%}$ & $\mathbf{4.0 \cdot 10^{5}}$ & $1.5 \cdot 10^{6} {~}^{+6.0 \cdot 10^{5}}_{-5.6 \cdot 10^{5}}$ & $2.4 \cdot 10^{-1} {~}^{+1.7 \cdot 10^{-2}}_{-1.7 \cdot 10^{-2}}$\\

\nopagebreak
 & $\mathrm{NAC}_{+}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{Gated~}^{\mathrm{NAU}}_{\mathrm{NMU}}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & Linear & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NALU & $2\% {~}^{+5\%}_{-1\%}$ & $2.6 \cdot 10^{6}$ & $3.3 \cdot 10^{6} {~}^{+1.8 \cdot 10^{6}}_{-2.2 \cdot 10^{6}}$ & $5.0 \cdot 10^{-1} {~}^{+2.5 \cdot 10^{-6}}_{-8.0 \cdot 10^{-6}}$\\

\nopagebreak
 & NAU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NMU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & ReLU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
\multirow{-11}{*}{\centering\arraybackslash $\sqrt{z}$} & ReLU6 & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\
\cmidrule{1-6}
 & $\mathrm{NAC}_{\bullet,\mathrm{NMU}}$ & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $1.4 \cdot 10^{6}$ & $1.5 \cdot 10^{6} {~}^{+8.4 \cdot 10^{4}}_{-7.9 \cdot 10^{4}}$ & $\mathbf{2.9 \cdot 10^{-7}} {~}^{+1.4 \cdot 10^{-8}}_{-1.4 \cdot 10^{-8}}$\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet,\sigma}$ & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $1.9 \cdot 10^{6}$ & $1.9 \cdot 10^{6} {~}^{+5.3 \cdot 10^{4}}_{-6.2 \cdot 10^{4}}$ & $1.8 \cdot 10^{-2} {~}^{+4.3 \cdot 10^{-4}}_{-4.3 \cdot 10^{-4}}$\\

\nopagebreak
 & $\mathrm{NAC}_{\bullet}$ & $77\% {~}^{+7\%}_{-9\%}$ & $3.3 \cdot 10^{6}$ & $3.2 \cdot 10^{6} {~}^{+1.6 \cdot 10^{5}}_{-2.0 \cdot 10^{5}}$ & $1.8 \cdot 10^{-2} {~}^{+5.8 \cdot 10^{-4}}_{-5.7 \cdot 10^{-4}}$\\

\nopagebreak
 & $\mathrm{NAC}_{+}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & $\mathrm{Gated~}^{\mathrm{NAU}}_{\mathrm{NMU}}$ & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & Linear & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NALU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NAU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
 & NMU & $\mathbf{100\%} {~}^{+0\%}_{-4\%}$ & $\mathbf{1.2 \cdot 10^{6}}$ & $\mathbf{1.3 \cdot 10^{6}} {~}^{+3.1 \cdot 10^{4}}_{-3.6 \cdot 10^{4}}$ & $3.7 \cdot 10^{-5} {~}^{+5.4 \cdot 10^{-5}}_{-3.7 \cdot 10^{-5}}$\\

\nopagebreak
 & ReLU & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\

\nopagebreak
\multirow{-11}{*}{\centering\arraybackslash $z^2$} & ReLU6 & $0\% {~}^{+4\%}_{-0\%}$ & --- & --- & ---\\*
\end{longtable}